{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "tutorial-wages",
   "metadata": {},
   "source": [
    "## Description：\n",
    "这个jupyter是AutoInt模型的demo， 主要包括读取数据， 搭建模型，模型训练和评估三大部分。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "permanent-effort",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T14:36:31.304869Z",
     "start_time": "2021-05-07T14:36:27.703217Z"
    }
   },
   "outputs": [],
   "source": [
    "# python基础包\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# 特征处理与数据集划分\n",
    "from sklearn.preprocessing import OneHotEncoder, MinMaxScaler, StandardScaler, LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "from utils import DenseFeat, SparseFeat, VarLenSparseFeat\n",
    "\n",
    "# 导入模型\n",
    "from AutoInt import AutoInt\n",
    "\n",
    "# 模型训练相关\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\n",
    "from tensorflow.keras.metrics import AUC\n",
    "from tensorflow.keras.losses import binary_crossentropy\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "\n",
    "# 一些相关设置\n",
    "import warnings\n",
    "plt.style.use('fivethirtyeight')\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eleven-given",
   "metadata": {},
   "source": [
    "## 导入数据与初步处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "employed-scanner",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T14:36:32.610405Z",
     "start_time": "2021-05-07T14:36:32.600432Z"
    }
   },
   "outputs": [],
   "source": [
    "# 简单处理特征， 包括填充缺失值，数值处理， 类别编码\n",
    "def data_process(data_df, dense_features, sparse_features):\n",
    "    # 连续型特征用0填充缺失, 并进行对数转换\n",
    "    data_df[dense_features] = data_df[dense_features].fillna(0.0)\n",
    "    for f in dense_features:\n",
    "        data_df[f] = data_df[f].apply(lambda x: np.log(x+1) if x > -1 else -1)\n",
    "    \n",
    "    # 离散型特征用-1填充，并进行LabelEncoder编码\n",
    "    data_df[sparse_features] = data_df[sparse_features].fillna(\"-1\")\n",
    "    for f in sparse_features:\n",
    "        lbe = LabelEncoder()\n",
    "        data_df[f] = lbe.fit_transform(data_df[f])\n",
    "    \n",
    "    return data_df[dense_features+sparse_features]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "unlimited-moral",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T14:36:34.094491Z",
     "start_time": "2021-05-07T14:36:34.073232Z"
    }
   },
   "outputs": [],
   "source": [
    "\"\"\"读取数据\"\"\"\n",
    "data = pd.read_csv('data/criteo_sample.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "synthetic-finnish",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T14:36:34.783516Z",
     "start_time": "2021-05-07T14:36:34.764494Z"
    }
   },
   "outputs": [],
   "source": [
    "\"\"\"划分dense和sparse特征\"\"\"\n",
    "columns = data.columns.values\n",
    "dense_features = [feat for feat in columns if 'I' in feat]\n",
    "sparse_features = [feat for feat in columns if 'C' in feat]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "skilled-withdrawal",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T14:36:36.124351Z",
     "start_time": "2021-05-07T14:36:36.083461Z"
    }
   },
   "outputs": [],
   "source": [
    "\"\"\"简单的数据预处理\"\"\"\n",
    "train_data = data_process(data, dense_features, sparse_features)\n",
    "train_data['label'] = data['label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "geological-stuart",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T14:36:41.167522Z",
     "start_time": "2021-05-07T14:36:41.157548Z"
    }
   },
   "outputs": [],
   "source": [
    "\"\"\"划分数据集\"\"\"\n",
    "trnx, valx, trny, valy = train_test_split(train_data[dense_features+sparse_features], train_data['label'], test_size=0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "combined-lesbian",
   "metadata": {},
   "source": [
    "## 模型建立"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "twelve-tamil",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T14:36:43.193917Z",
     "start_time": "2021-05-07T14:36:43.167949Z"
    }
   },
   "outputs": [],
   "source": [
    "\"\"\"特征分组， 分成linear部分和DNN部分(根据实际场景进行选择)，并将分组之后的特征用namedtuple封装起来(DenseFeat, SparseFeat)\"\"\"\n",
    "linear_feature_columns = [SparseFeat(feat, vocabulary_size=data[feat].nunique(), embedding_dim=4)\n",
    "                         for i, feat in enumerate(sparse_features)] + \\\n",
    "                         [DenseFeat(feat, 1, ) for feat in dense_features]\n",
    "\n",
    "dnn_feature_columns = [SparseFeat(feat, vocabulary_size=data[feat].nunique(), embedding_dim=4)\n",
    "                         for i, feat in enumerate(sparse_features)] + \\\n",
    "                         [DenseFeat(feat, 1, ) for feat in dense_features]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "motivated-photography",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T14:36:46.182643Z",
     "start_time": "2021-05-07T14:36:44.522303Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <bound method InteractingLayer.call of <AutoInt.InteractingLayer object at 0x000002283C236DC8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method InteractingLayer.call of <AutoInt.InteractingLayer object at 0x000002283C236DC8>>: AttributeError: module 'gast' has no attribute 'Str'\n",
      "WARNING: Entity <bound method InteractingLayer.call of <AutoInt.InteractingLayer object at 0x000002283C236DC8>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method InteractingLayer.call of <AutoInt.InteractingLayer object at 0x000002283C236DC8>>: AttributeError: module 'gast' has no attribute 'Str'\n",
      "WARNING:tensorflow:Entity <bound method InteractingLayer.call of <AutoInt.InteractingLayer object at 0x000002283C25D848>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method InteractingLayer.call of <AutoInt.InteractingLayer object at 0x000002283C25D848>>: AttributeError: module 'gast' has no attribute 'Str'\n",
      "WARNING: Entity <bound method InteractingLayer.call of <AutoInt.InteractingLayer object at 0x000002283C25D848>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method InteractingLayer.call of <AutoInt.InteractingLayer object at 0x000002283C25D848>>: AttributeError: module 'gast' has no attribute 'Str'\n",
      "WARNING:tensorflow:Entity <bound method InteractingLayer.call of <AutoInt.InteractingLayer object at 0x000002283C2B8848>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method InteractingLayer.call of <AutoInt.InteractingLayer object at 0x000002283C2B8848>>: AttributeError: module 'gast' has no attribute 'Str'\n",
      "WARNING: Entity <bound method InteractingLayer.call of <AutoInt.InteractingLayer object at 0x000002283C2B8848>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method InteractingLayer.call of <AutoInt.InteractingLayer object at 0x000002283C2B8848>>: AttributeError: module 'gast' has no attribute 'Str'\n",
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "C1 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C2 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C3 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C4 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C5 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C6 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C7 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C8 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C9 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C10 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C11 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C12 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C13 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C14 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C15 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C16 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C17 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C18 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C19 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C20 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C21 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C22 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C23 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C24 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C25 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C26 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C1 (Embedding)           (None, 1, 4)         108         C1[0][0]                         \n",
      "                                                                 C1[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C2 (Embedding)           (None, 1, 4)         368         C2[0][0]                         \n",
      "                                                                 C2[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C3 (Embedding)           (None, 1, 4)         688         C3[0][0]                         \n",
      "                                                                 C3[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C4 (Embedding)           (None, 1, 4)         628         C4[0][0]                         \n",
      "                                                                 C4[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C5 (Embedding)           (None, 1, 4)         48          C5[0][0]                         \n",
      "                                                                 C5[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C6 (Embedding)           (None, 1, 4)         28          C6[0][0]                         \n",
      "                                                                 C6[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C7 (Embedding)           (None, 1, 4)         732         C7[0][0]                         \n",
      "                                                                 C7[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C8 (Embedding)           (None, 1, 4)         76          C8[0][0]                         \n",
      "                                                                 C8[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C9 (Embedding)           (None, 1, 4)         8           C9[0][0]                         \n",
      "                                                                 C9[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C10 (Embedding)          (None, 1, 4)         568         C10[0][0]                        \n",
      "                                                                 C10[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C11 (Embedding)          (None, 1, 4)         692         C11[0][0]                        \n",
      "                                                                 C11[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C12 (Embedding)          (None, 1, 4)         680         C12[0][0]                        \n",
      "                                                                 C12[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C13 (Embedding)          (None, 1, 4)         664         C13[0][0]                        \n",
      "                                                                 C13[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C14 (Embedding)          (None, 1, 4)         56          C14[0][0]                        \n",
      "                                                                 C14[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C15 (Embedding)          (None, 1, 4)         680         C15[0][0]                        \n",
      "                                                                 C15[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C16 (Embedding)          (None, 1, 4)         672         C16[0][0]                        \n",
      "                                                                 C16[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C17 (Embedding)          (None, 1, 4)         36          C17[0][0]                        \n",
      "                                                                 C17[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C18 (Embedding)          (None, 1, 4)         508         C18[0][0]                        \n",
      "                                                                 C18[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C19 (Embedding)          (None, 1, 4)         176         C19[0][0]                        \n",
      "                                                                 C19[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C20 (Embedding)          (None, 1, 4)         16          C20[0][0]                        \n",
      "                                                                 C20[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C21 (Embedding)          (None, 1, 4)         676         C21[0][0]                        \n",
      "                                                                 C21[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C22 (Embedding)          (None, 1, 4)         24          C22[0][0]                        \n",
      "                                                                 C22[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C23 (Embedding)          (None, 1, 4)         40          C23[0][0]                        \n",
      "                                                                 C23[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C24 (Embedding)          (None, 1, 4)         500         C24[0][0]                        \n",
      "                                                                 C24[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C25 (Embedding)          (None, 1, 4)         80          C25[0][0]                        \n",
      "                                                                 C25[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "kd_emb_C26 (Embedding)          (None, 1, 4)         360         C26[0][0]                        \n",
      "                                                                 C26[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 26, 4)        0           kd_emb_C1[0][0]                  \n",
      "                                                                 kd_emb_C2[0][0]                  \n",
      "                                                                 kd_emb_C3[0][0]                  \n",
      "                                                                 kd_emb_C4[0][0]                  \n",
      "                                                                 kd_emb_C5[0][0]                  \n",
      "                                                                 kd_emb_C6[0][0]                  \n",
      "                                                                 kd_emb_C7[0][0]                  \n",
      "                                                                 kd_emb_C8[0][0]                  \n",
      "                                                                 kd_emb_C9[0][0]                  \n",
      "                                                                 kd_emb_C10[0][0]                 \n",
      "                                                                 kd_emb_C11[0][0]                 \n",
      "                                                                 kd_emb_C12[0][0]                 \n",
      "                                                                 kd_emb_C13[0][0]                 \n",
      "                                                                 kd_emb_C14[0][0]                 \n",
      "                                                                 kd_emb_C15[0][0]                 \n",
      "                                                                 kd_emb_C16[0][0]                 \n",
      "                                                                 kd_emb_C17[0][0]                 \n",
      "                                                                 kd_emb_C18[0][0]                 \n",
      "                                                                 kd_emb_C19[0][0]                 \n",
      "                                                                 kd_emb_C20[0][0]                 \n",
      "                                                                 kd_emb_C21[0][0]                 \n",
      "                                                                 kd_emb_C22[0][0]                 \n",
      "                                                                 kd_emb_C23[0][0]                 \n",
      "                                                                 kd_emb_C24[0][0]                 \n",
      "                                                                 kd_emb_C25[0][0]                 \n",
      "                                                                 kd_emb_C26[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "interacting_layer (InteractingL (None, 26, 16)       256         concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "interacting_layer_1 (Interactin (None, 26, 16)       1024        interacting_layer[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "I1 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I2 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I3 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I4 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I5 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I6 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I7 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I8 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I9 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I10 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I11 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I12 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I13 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_27 (Flatten)            (None, 4)            0           kd_emb_C1[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_28 (Flatten)            (None, 4)            0           kd_emb_C2[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_29 (Flatten)            (None, 4)            0           kd_emb_C3[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_30 (Flatten)            (None, 4)            0           kd_emb_C4[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_31 (Flatten)            (None, 4)            0           kd_emb_C5[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_32 (Flatten)            (None, 4)            0           kd_emb_C6[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_33 (Flatten)            (None, 4)            0           kd_emb_C7[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_34 (Flatten)            (None, 4)            0           kd_emb_C8[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_35 (Flatten)            (None, 4)            0           kd_emb_C9[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_36 (Flatten)            (None, 4)            0           kd_emb_C10[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_37 (Flatten)            (None, 4)            0           kd_emb_C11[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_38 (Flatten)            (None, 4)            0           kd_emb_C12[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_39 (Flatten)            (None, 4)            0           kd_emb_C13[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_40 (Flatten)            (None, 4)            0           kd_emb_C14[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_41 (Flatten)            (None, 4)            0           kd_emb_C15[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_42 (Flatten)            (None, 4)            0           kd_emb_C16[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_43 (Flatten)            (None, 4)            0           kd_emb_C17[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_44 (Flatten)            (None, 4)            0           kd_emb_C18[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_45 (Flatten)            (None, 4)            0           kd_emb_C19[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_46 (Flatten)            (None, 4)            0           kd_emb_C20[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_47 (Flatten)            (None, 4)            0           kd_emb_C21[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_48 (Flatten)            (None, 4)            0           kd_emb_C22[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_49 (Flatten)            (None, 4)            0           kd_emb_C23[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_50 (Flatten)            (None, 4)            0           kd_emb_C24[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_51 (Flatten)            (None, 4)            0           kd_emb_C25[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_52 (Flatten)            (None, 4)            0           kd_emb_C26[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "interacting_layer_2 (Interactin (None, 26, 16)       1024        interacting_layer_1[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 13)           0           I1[0][0]                         \n",
      "                                                                 I2[0][0]                         \n",
      "                                                                 I3[0][0]                         \n",
      "                                                                 I4[0][0]                         \n",
      "                                                                 I5[0][0]                         \n",
      "                                                                 I6[0][0]                         \n",
      "                                                                 I7[0][0]                         \n",
      "                                                                 I8[0][0]                         \n",
      "                                                                 I9[0][0]                         \n",
      "                                                                 I10[0][0]                        \n",
      "                                                                 I11[0][0]                        \n",
      "                                                                 I12[0][0]                        \n",
      "                                                                 I13[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)     (None, 104)          0           flatten_27[0][0]                 \n",
      "                                                                 flatten_28[0][0]                 \n",
      "                                                                 flatten_29[0][0]                 \n",
      "                                                                 flatten_30[0][0]                 \n",
      "                                                                 flatten_31[0][0]                 \n",
      "                                                                 flatten_32[0][0]                 \n",
      "                                                                 flatten_33[0][0]                 \n",
      "                                                                 flatten_34[0][0]                 \n",
      "                                                                 flatten_35[0][0]                 \n",
      "                                                                 flatten_36[0][0]                 \n",
      "                                                                 flatten_37[0][0]                 \n",
      "                                                                 flatten_38[0][0]                 \n",
      "                                                                 flatten_39[0][0]                 \n",
      "                                                                 flatten_40[0][0]                 \n",
      "                                                                 flatten_41[0][0]                 \n",
      "                                                                 flatten_42[0][0]                 \n",
      "                                                                 flatten_43[0][0]                 \n",
      "                                                                 flatten_44[0][0]                 \n",
      "                                                                 flatten_45[0][0]                 \n",
      "                                                                 flatten_46[0][0]                 \n",
      "                                                                 flatten_47[0][0]                 \n",
      "                                                                 flatten_48[0][0]                 \n",
      "                                                                 flatten_49[0][0]                 \n",
      "                                                                 flatten_50[0][0]                 \n",
      "                                                                 flatten_51[0][0]                 \n",
      "                                                                 flatten_52[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_26 (Flatten)            (None, 416)          0           interacting_layer_2[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_4 (Concatenate)     (None, 533)          0           concatenate_2[0][0]              \n",
      "                                                                 concatenate_3[0][0]              \n",
      "                                                                 flatten_26[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 1024)         546816      concatenate_4[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               multiple             0           dense_1[0][0]                    \n",
      "                                                                 dense_2[0][0]                    \n",
      "                                                                 dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 512)          524800      dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C1 (Embedding)           (None, 1, 1)         27          C1[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C2 (Embedding)           (None, 1, 1)         92          C2[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C3 (Embedding)           (None, 1, 1)         172         C3[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C4 (Embedding)           (None, 1, 1)         157         C4[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C5 (Embedding)           (None, 1, 1)         12          C5[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C6 (Embedding)           (None, 1, 1)         7           C6[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C7 (Embedding)           (None, 1, 1)         183         C7[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C8 (Embedding)           (None, 1, 1)         19          C8[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C9 (Embedding)           (None, 1, 1)         2           C9[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C10 (Embedding)          (None, 1, 1)         142         C10[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C11 (Embedding)          (None, 1, 1)         173         C11[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C12 (Embedding)          (None, 1, 1)         170         C12[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C13 (Embedding)          (None, 1, 1)         166         C13[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C14 (Embedding)          (None, 1, 1)         14          C14[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C15 (Embedding)          (None, 1, 1)         170         C15[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C16 (Embedding)          (None, 1, 1)         168         C16[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C17 (Embedding)          (None, 1, 1)         9           C17[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C18 (Embedding)          (None, 1, 1)         127         C18[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C19 (Embedding)          (None, 1, 1)         44          C19[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C20 (Embedding)          (None, 1, 1)         4           C20[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C21 (Embedding)          (None, 1, 1)         169         C21[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C22 (Embedding)          (None, 1, 1)         6           C22[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C23 (Embedding)          (None, 1, 1)         10          C23[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C24 (Embedding)          (None, 1, 1)         125         C24[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C25 (Embedding)          (None, 1, 1)         20          C25[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "1d_emb_C26 (Embedding)          (None, 1, 1)         90          C26[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 13)           0           I1[0][0]                         \n",
      "                                                                 I2[0][0]                         \n",
      "                                                                 I3[0][0]                         \n",
      "                                                                 I4[0][0]                         \n",
      "                                                                 I5[0][0]                         \n",
      "                                                                 I6[0][0]                         \n",
      "                                                                 I7[0][0]                         \n",
      "                                                                 I8[0][0]                         \n",
      "                                                                 I9[0][0]                         \n",
      "                                                                 I10[0][0]                        \n",
      "                                                                 I11[0][0]                        \n",
      "                                                                 I12[0][0]                        \n",
      "                                                                 I13[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "flatten (Flatten)               (None, 1)            0           1d_emb_C1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 1)            0           1d_emb_C2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_2 (Flatten)             (None, 1)            0           1d_emb_C3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_3 (Flatten)             (None, 1)            0           1d_emb_C4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_4 (Flatten)             (None, 1)            0           1d_emb_C5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_5 (Flatten)             (None, 1)            0           1d_emb_C6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_6 (Flatten)             (None, 1)            0           1d_emb_C7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_7 (Flatten)             (None, 1)            0           1d_emb_C8[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_8 (Flatten)             (None, 1)            0           1d_emb_C9[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_9 (Flatten)             (None, 1)            0           1d_emb_C10[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_10 (Flatten)            (None, 1)            0           1d_emb_C11[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_11 (Flatten)            (None, 1)            0           1d_emb_C12[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_12 (Flatten)            (None, 1)            0           1d_emb_C13[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_13 (Flatten)            (None, 1)            0           1d_emb_C14[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_14 (Flatten)            (None, 1)            0           1d_emb_C15[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_15 (Flatten)            (None, 1)            0           1d_emb_C16[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_16 (Flatten)            (None, 1)            0           1d_emb_C17[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_17 (Flatten)            (None, 1)            0           1d_emb_C18[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_18 (Flatten)            (None, 1)            0           1d_emb_C19[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_19 (Flatten)            (None, 1)            0           1d_emb_C20[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_20 (Flatten)            (None, 1)            0           1d_emb_C21[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_21 (Flatten)            (None, 1)            0           1d_emb_C22[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_22 (Flatten)            (None, 1)            0           1d_emb_C23[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_23 (Flatten)            (None, 1)            0           1d_emb_C24[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_24 (Flatten)            (None, 1)            0           1d_emb_C25[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_25 (Flatten)            (None, 1)            0           1d_emb_C26[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 256)          131328      dropout[1][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 1)            14          concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "add (Add)                       (None, 1)            0           flatten[0][0]                    \n",
      "                                                                 flatten_1[0][0]                  \n",
      "                                                                 flatten_2[0][0]                  \n",
      "                                                                 flatten_3[0][0]                  \n",
      "                                                                 flatten_4[0][0]                  \n",
      "                                                                 flatten_5[0][0]                  \n",
      "                                                                 flatten_6[0][0]                  \n",
      "                                                                 flatten_7[0][0]                  \n",
      "                                                                 flatten_8[0][0]                  \n",
      "                                                                 flatten_9[0][0]                  \n",
      "                                                                 flatten_10[0][0]                 \n",
      "                                                                 flatten_11[0][0]                 \n",
      "                                                                 flatten_12[0][0]                 \n",
      "                                                                 flatten_13[0][0]                 \n",
      "                                                                 flatten_14[0][0]                 \n",
      "                                                                 flatten_15[0][0]                 \n",
      "                                                                 flatten_16[0][0]                 \n",
      "                                                                 flatten_17[0][0]                 \n",
      "                                                                 flatten_18[0][0]                 \n",
      "                                                                 flatten_19[0][0]                 \n",
      "                                                                 flatten_20[0][0]                 \n",
      "                                                                 flatten_21[0][0]                 \n",
      "                                                                 flatten_22[0][0]                 \n",
      "                                                                 flatten_23[0][0]                 \n",
      "                                                                 flatten_24[0][0]                 \n",
      "                                                                 flatten_25[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 1)            0           dense[0][0]                      \n",
      "                                                                 add[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 1)            257         dropout[2][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, 1)            0           add_1[0][0]                      \n",
      "                                                                 dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 1)            2           add_2[0][0]                      \n",
      "==================================================================================================\n",
      "Total params: 1,216,911\n",
      "Trainable params: 1,216,911\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "\"\"\"构建AutoInt模型\"\"\"\n",
    "model = AutoInt(linear_feature_columns, dnn_feature_columns)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "catholic-beast",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T14:37:06.987683Z",
     "start_time": "2021-05-07T14:37:06.974718Z"
    }
   },
   "outputs": [],
   "source": [
    "\"\"\"设置超参数\"\"\"\n",
    "learning_rate = 0.001\n",
    "batch_size = 64\n",
    "epochs = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "developing-niagara",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T14:37:08.471518Z",
     "start_time": "2021-05-07T14:37:08.450613Z"
    }
   },
   "outputs": [],
   "source": [
    "\"\"\"模型编译\"\"\"\n",
    "model.compile(loss=binary_crossentropy, optimizer=Adam(learning_rate=learning_rate),\n",
    "                      metrics=['binary_crossentropy', AUC(name='auc')])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "disabled-rainbow",
   "metadata": {},
   "source": [
    "## 模型的训练与评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "emotional-footwear",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T14:37:32.160991Z",
     "start_time": "2021-05-07T14:37:24.407920Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From D:\\anaconda3\\envs\\tfenv\\lib\\site-packages\\tensorflow\\python\\ops\\math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "Train on 144 samples, validate on 36 samples\n",
      "Epoch 1/20\n",
      "144/144 [==============================] - 7s 47ms/sample - loss: 3.6515 - binary_crossentropy: 3.6515 - auc: 0.5132 - val_loss: 0.9130 - val_binary_crossentropy: 0.9130 - val_auc: 0.5692\n",
      "Epoch 2/20\n",
      "144/144 [==============================] - 0s 1ms/sample - loss: 1.4216 - binary_crossentropy: 1.4216 - auc: 0.5153 - val_loss: 1.2133 - val_binary_crossentropy: 1.2133 - val_auc: 0.5491\n",
      "Epoch 3/20\n",
      "144/144 [==============================] - 0s 1ms/sample - loss: 1.1401 - binary_crossentropy: 1.1401 - auc: 0.5676 - val_loss: 0.6998 - val_binary_crossentropy: 0.6998 - val_auc: 0.5759\n",
      "Epoch 4/20\n",
      "144/144 [==============================] - 0s 1ms/sample - loss: 0.9093 - binary_crossentropy: 0.9093 - auc: 0.5995 - val_loss: 0.6940 - val_binary_crossentropy: 0.6940 - val_auc: 0.5871\n",
      "Epoch 5/20\n",
      "144/144 [==============================] - 0s 1ms/sample - loss: 0.7556 - binary_crossentropy: 0.7556 - auc: 0.6398 - val_loss: 0.7629 - val_binary_crossentropy: 0.7629 - val_auc: 0.5804\n",
      "Epoch 6/20\n",
      "144/144 [==============================] - 0s 1ms/sample - loss: 0.5729 - binary_crossentropy: 0.5729 - auc: 0.7653 - val_loss: 0.7615 - val_binary_crossentropy: 0.7615 - val_auc: 0.5268\n",
      "Epoch 7/20\n",
      "144/144 [==============================] - 0s 1ms/sample - loss: 0.5535 - binary_crossentropy: 0.5535 - auc: 0.7374 - val_loss: 0.7466 - val_binary_crossentropy: 0.7466 - val_auc: 0.4978\n",
      "Epoch 8/20\n",
      "144/144 [==============================] - 0s 1ms/sample - loss: 0.5152 - binary_crossentropy: 0.5152 - auc: 0.7652 - val_loss: 0.8050 - val_binary_crossentropy: 0.8050 - val_auc: 0.4621\n",
      "Epoch 9/20\n",
      "144/144 [==============================] - 0s 1ms/sample - loss: 0.4892 - binary_crossentropy: 0.4892 - auc: 0.8040 - val_loss: 0.9195 - val_binary_crossentropy: 0.9195 - val_auc: 0.4152\n",
      "Epoch 10/20\n",
      "144/144 [==============================] - 0s 1ms/sample - loss: 0.4453 - binary_crossentropy: 0.4453 - auc: 0.8346 - val_loss: 0.9765 - val_binary_crossentropy: 0.9765 - val_auc: 0.4085\n",
      "Epoch 11/20\n",
      "144/144 [==============================] - 0s 1ms/sample - loss: 0.4278 - binary_crossentropy: 0.4278 - auc: 0.8449 - val_loss: 0.9675 - val_binary_crossentropy: 0.9675 - val_auc: 0.4085\n",
      "Epoch 12/20\n",
      "144/144 [==============================] - 0s 1ms/sample - loss: 0.3837 - binary_crossentropy: 0.3837 - auc: 0.8719 - val_loss: 0.8837 - val_binary_crossentropy: 0.8837 - val_auc: 0.4330\n",
      "Epoch 13/20\n",
      "144/144 [==============================] - 0s 1ms/sample - loss: 0.4592 - binary_crossentropy: 0.4592 - auc: 0.8257 - val_loss: 0.8155 - val_binary_crossentropy: 0.8155 - val_auc: 0.4732\n",
      "Epoch 14/20\n",
      "144/144 [==============================] - 0s 1ms/sample - loss: 0.3806 - binary_crossentropy: 0.3806 - auc: 0.8845 - val_loss: 0.8238 - val_binary_crossentropy: 0.8238 - val_auc: 0.4598\n",
      "Epoch 15/20\n",
      "144/144 [==============================] - 0s 1ms/sample - loss: 0.3418 - binary_crossentropy: 0.3418 - auc: 0.8965 - val_loss: 0.7767 - val_binary_crossentropy: 0.7767 - val_auc: 0.5000\n",
      "Epoch 16/20\n",
      "144/144 [==============================] - 0s 1ms/sample - loss: 0.3208 - binary_crossentropy: 0.3208 - auc: 0.9106 - val_loss: 0.6877 - val_binary_crossentropy: 0.6877 - val_auc: 0.5513\n",
      "Epoch 17/20\n",
      "144/144 [==============================] - 0s 1ms/sample - loss: 0.2549 - binary_crossentropy: 0.2549 - auc: 0.9465 - val_loss: 0.6878 - val_binary_crossentropy: 0.6878 - val_auc: 0.5446\n",
      "Epoch 18/20\n",
      "144/144 [==============================] - 0s 1ms/sample - loss: 0.2259 - binary_crossentropy: 0.2259 - auc: 0.9662 - val_loss: 0.8191 - val_binary_crossentropy: 0.8191 - val_auc: 0.5045\n",
      "Epoch 19/20\n",
      "144/144 [==============================] - 0s 1ms/sample - loss: 0.1867 - binary_crossentropy: 0.1867 - auc: 0.9815 - val_loss: 0.8564 - val_binary_crossentropy: 0.8564 - val_auc: 0.4821\n",
      "Epoch 20/20\n",
      "144/144 [==============================] - 0s 1ms/sample - loss: 0.1272 - binary_crossentropy: 0.1272 - auc: 0.9986 - val_loss: 0.7963 - val_binary_crossentropy: 0.7963 - val_auc: 0.5089\n"
     ]
    }
   ],
   "source": [
    "\"\"\"模型训练\"\"\"\n",
    "callbacks = [\n",
    "    EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True),   # 早停\n",
    "    ReduceLROnPlateau(monitor='val_loss', patience=3, factor=0.01, verbose=1)\n",
    "]\n",
    "#将输入数据转化成字典的形式输入\n",
    "trnx_model_input = {name: trnx[name] for name in dense_features + sparse_features}\n",
    "valx_model_input = {name: valx[name] for name in dense_features + sparse_features}\n",
    "\n",
    "history = model.fit(\n",
    "                    trnx_model_input, \n",
    "                    trny.values,\n",
    "                    epochs=epochs,\n",
    "                    #callbacks=callbacks,  # checkpoint\n",
    "                    batch_size=batch_size,\n",
    "                    validation_split=0.2\n",
    "                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "together-marks",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T14:37:47.623625Z",
     "start_time": "2021-05-07T14:37:47.478971Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbUAAAE0CAYAAACrRq2gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABNz0lEQVR4nO3dd3hUZf7//+eZlh4mPZQktFBCb0FgpQgLitJcWCxrYd0fqKjr2hddBBuwlpUFQb+y+FEBVwSRooKrdKVJFwVCSagpJKRnMpmZ8/tjwsCk15lJ8n5c11yZU+eek8m8cp9z3/dRMjMzVYQQQohGQOPuAgghhBB1RUJNCCFEoyGhJoQQotGQUBNCCNFoSKgJIYRoNCTUhBBCNBoSakJ4kKSkJIxGI4888ohL97Njxw6MRiNz5syp1esK4W4SaqJJMxqNGI1GgoKCOHv2bLnrjR8/3rHu0qVLXVhCIUR1SKiJJk+n06GqKp988kmZyxMTE9m2bRs6nc7FJRNCVJeEmmjygoOD6devHytWrMBisZRa/umnn6KqKrfeeqsbSieEqA4JNSGA+++/n5SUFL799lun+RaLheXLl9OnTx+6dOlS7vaJiYk8+uijxMXFERYWRmxsLA8++CC//PJLmevn5OQwY8YM4uLiiIiIoF+/fixcuBBVLX/UOpPJxIIFCxgyZAgtW7akRYsWDB06lKVLl1a4XW1V572ZzWY++OADhgwZQps2bYiMjKRr165MnDiRdevWOa37yy+/8Je//IXu3bsTERFB27ZtGThwIE8//TRZWVn19n5E4ybnU4QA7rzzTmbMmMEnn3zCmDFjHPM3bdpEcnIyM2bM4OLFi2Vue+jQIcaNG0d2djYjR46kS5cunD17lvXr17Nx40ZWrFjBLbfc4li/sLCQcePGceDAAeLi4pg0aRLZ2dm89dZb/Pjjj2W+Rk5ODuPHj2f//v10796de+65B4AffviBp556in379rF48eI6PCI1e2+PPvooq1atolOnTkyaNAk/Pz8uX77MgQMH2LBhA2PHjgXsgTZixAgURWHUqFG0adOG3Nxczp07x4oVK5g+fTrNmjWr8/cjGj8JNSEAPz8/Jk6cyMcff8z58+eJiooC4JNPPsHf358777yTBQsWlNpOVVUefvhhsrKyWLRokSNsALZu3cqECROYOnUqR44cwdfXF4CFCxdy4MABRo8ezbJly9Bo7CdM/va3vzF06NAyyzdjxgz279/PrFmzePLJJx3zCwsLue+++/jss88YO3Yst912Wx0dkeq/t6ysLFavXk3Pnj35/vvvS12DTE9Pdzz/7LPPMJlMLFu2jDvuuMNpvZycHAwGQ529D9G0yOlHIYo98MAD2Gw2li1bBsDFixf5/vvv+cMf/oC/v3+Z2+zZs4fjx4/Tu3dvpy99gKFDh3LHHXdw5coVvvnmG8f85cuXoygKs2fPdgQaQHR0NNOmTSv1GlevXuWzzz6je/fuToEG4OXlxcyZMwH4/PPPa/S+y1Pd96YoCqqqYjAY0Gq1pfYXEhJSap6Pj0+peQEBAXh5edXRuxBNjdTUhCjWs2dPunfvzvLly3nuuef49NNPsVqtPPDAA+Vuc/jwYQAGDx5c5vKhQ4eyfv16Dh8+zMSJE8nJyeHMmTNERkYSGxtbav1BgwaVmrd//34sFgsajabMfmTXGrecPHmySu+zqqr73gIDA7n11lvZuHEjgwYN4o477mDAgAH069ev1D8Fd955J++//z733nsvY8eOZfDgwcTHx9OhQ4c6fQ+i6ZFQE+IGDzzwAE8//TSbNm1i2bJldO3ald69e5e7fnZ2NgDh4eFlLo+IiABwNHy4tn5YWFiZ65e1n4yMDMB+fevQoUPlliU3N7fcZTVR3fcG8NFHH/Hvf/+bVatW8c9//hMAvV7PrbfeymuvvUZMTAwAffr0YePGjbz99tts2LCBlStXAvba6pNPPsmf//znOn0voumQ049C3GDSpEn4+vry7LPPcuHCBR588MEK1w8MDAQgNTW1zOUpKSlO6137mZaWVub6Ze3n2jZTp04lMzOz3MeRI0cqf4PVUN33BvbTic8//zz79u3jt99+Y+nSpYwYMYL169czceJEioqKHOv269eP//73vyQmJvL999/z4osvYjKZeOqpp/jss8/q9L2IpkNCTYgbBAYGMmHCBC5evIivry+TJk2qcP0ePXoA9mGmyrJt2zbAfmoT7NeL2rZtS0pKCqdOnSq1flmtH/v27YtGo2HXrl3VeSu1Vt33VlLz5s258847+eyzz4iPjychIYHjx4+XWs9gMNC3b1+effZZ3n//fQA2bNhQB+9ANEUSakKUMGPGDJYtW8aqVasqbVbev39/OnbsyP79+0s11Ni2bRvr168nJCSE0aNHO+bfe++9qKrKzJkzsdlsjvnnzp3jgw8+KPUaoaGhTJ48maNHjzJnzpwyO4hfvHixzq+pVfe9Xblypcy+a4WFhY5TlNdagO7Zs4eCgoJS616r/V1bT4jqkmtqQpTQsmVLWrZsWaV1FUVh8eLFjB8/nocffpg1a9Y4+nKtW7cOg8HA+++/7/Ql/dhjj/H111/zzTffcPPNNzNixAiys7NZs2YNAwYMKNUBHOCf//wnZ86cYd68eXz++ecMHDiQiIgIR41v3759vP7663Xa0KK67+3SpUsMHjyYuLg4unTpQsuWLcnLy2Pz5s2cPn2asWPH0q5dOwDmz5/P9u3bGTBgADExMQQEBHDq1Ck2bdqEj49PrQd0Fk2XhJoQtdS7d2+2bt3Km2++ydatW/nhhx9o1qwZt99+O08//TTdu3d3Wt/Ly4uvvvqKuXPnsmbNGt5//32io6N5+umnGTNmTJmhFhAQwIYNG/j000/54osv2LBhAyaTibCwMGJiYnj55ZeZMGGCW99bdHQ0M2bMYMeOHfz4449cuXKFZs2a0bZtW/761786dQv4y1/+QlBQEPv372fPnj0UFRXRvHlz7rrrLh577DFpBSlqTMnMzKy/8XWEEEIIF5JrakIIIRoNCTUhhBCNhoSaEEKIRkNCTQghRKMhoSaEEKLRkFATQgjRaEioCSGEaDQk1CqRkJDg7iI0CXKcXUOOs2vIcXYfCTUhhBCNhoSaEEKIRkNCTQghRKMhoSaEEKLRaNKj9FssFvLy8ipcx9vb2+l29aI0Pz8/dLom/VESQniIJvtNZLFYyMnJwWg0oihKuet5eXnh7e3twpI1LKqqkpmZSUBAgASbEMLtmuzpx7y8vEoDTVROURSMRmOlNV4hhHCFJv2vdUWBVmCxYbZCXpGC1WKhhZ8WrQRgmeQfAyGEp2jSoVaRM9lWzDYVUAAbod4afHTy5S2EEJ6syZ5+rIyX1nnabHVPOYQQQlSdhFo5DFrnWlmhTXVTSerfI488wuTJk91dDCGEqDU5/VgOL41zqJmt7g81o9FY4fK7776bxYsXV3u/c+fORVXd//6EEKK23BZqH374IR999BHnz58HoFOnTjzzzDOMGjWqzPWTkpLo0aNHqfmrVq1ixIgRdV4+Q4nTj55QUztx4oTj+aZNm3jiiSec5pXselBUVIRer690v82aNau7QgohhBu57fRjixYtmD17Ntu2bWPLli0MHjyYe++9l19++aXC7VavXs2JEyccj8GDB9dL+UrX1OrlZaolIiLC8bgWRNemTSYTMTExrFq1ijFjxhAZGclHH31ERkYGDz30EHFxcURGRnLTTTexbNkyp/2WPP14++238/TTT/PKK6/Qtm1b2rdvz0svvYTNZnPp+xVCiOpyW03t9ttvd5r+xz/+wX/+8x/27dtH165dy90uODiYiIiIeiuX8aOL9bbvsmROaVmn+5s9ezavvfYaCxYsQK/XYzKZ6NGjB3/9618JDAxk69at/O1vfyMqKoohQ4aUu58vvviCadOm8d1333H06FH+8pe/0LNnTyZOnFin5RVCiLrkEdfUrFYrX331FXl5ecTHx1e47n333YfJZKJdu3Y8+uijjBs3zkWlbBimTp1a6pg88cQTjucPPvgg27dvZ9WqVRWGWseOHXnxxRcBaN++PR9//DHbtm2TUBNCeDS3htqxY8cYOXIkJpMJPz8/li1bRpcuXcpc19/fn1dffZWbbroJnU7HN998w5QpU1i8eLG03LtBr169nKatViv/+te/+PLLL7l8+TJmsxmz2czvfve7CvdT8vcQGRlJWlpanZdXCCHqkltDLTY2lh07dpCdnc3atWt55JFH2LBhA3FxcaXWDQkJ4fHHH3dM9+rVi4yMDObPn19pqJV1F1pvb2+8vLxq/yZqyWQy1Wg7s9nstH1hYSEAOp3OaZ8LFizgvffe49VXX6Vz5874+fnxxhtvcOXKFcd6VqsVq9XqmLbZbCiK4rQfm82GxWIpt7zZ2dmkpqbW6L1cI3cLdg05zq4hx7n+xMbGlrvMraFmMBho27YtAD179uTAgQMsWrSIhQsXVmn7Pn36sHz58krXK+sAZGVllTlQ8Y3XuC7lWUgpuN44ItJHS3M/balt3MFgMADXWzxeC+iSAzD//PPP3Hrrrdx3332AfQDis2fP0qxZM8d6Wq0WrVbrmNZoNOh0Oqf9lFynpMDAQKKiomr8fhISEir8oIq6IcfZNeQ4u49Hdb622WyOGkhVHD16tF4bjTSGDtjt27dn+/bt7Nq1i5MnT/Lss89y7tw5dxdLCCHqhdtqarNmzWLkyJG0bNmS3NxcVq1axc6dO1m5ciVgb8W3f/9+1q1bB8CKFSvQ6/V0794djUbDxo0bWbJkCbNmzaq3MnpiB+zqevbZZ0lKSmLSpEl4e3tzzz33MGnSJI4fP+7uogkhRJ1zW6ilpKQwdepUUlNTCQwMpEuXLqxatYrhw4cDkJyczNmzZ522eeuttzh//jxarZZ27dqxcOHCem0kUqqm5kGhNm7cODIzMx3TMTExTtPXGI3GUv3SSio5CsnXX39d6TpCCOGJlMzMTM/5pnahrKysSkfSUFWVw+lF3HiAugfr0WpktP6SqnI8KyLXIFxDjrNryHF2H4+6puZpFEVBXyK/zA3wupoQQjQVEmqV0GucQ6zQA4bLEkIIUTYJtUoYShwhqakJIYTnklCrRMnTj57UWEQIIYQzCbVKlKqpSagJIYTHklCrRKlranL3FSGE8FgSapUwlGz9aFXlLtFCCOGhJNQqoVFAd0OwqUCR1NaEEMIjSahVgSePLCKEEOI6CbUq8CoRau5q1m80Git8PPLIIzXe95w5cxgwYEAdllYIIVzPI+587elKtoB0V03txIkTjuebNm3iiSeecJpX3m1hhBCiqZCaWhWUPP1odtM1tYiICMfj2jiLN8776aefGDJkCBEREXTv3p1XX33V6VY+69atY+DAgURGRtK6dWtGjx5Namoqy5cvZ968efz222+OWl9V7lMnhBCeRmpqJfg/MNR5GggFWtfT6+V+vLVO9vPDDz8wdepU5syZw6BBgzh//jxPPfUUhYWFvPbaa6SkpPDQQw8xc+ZMxo4dS15eHj///DMAd955J7/99hubNm1iw4YNgP2mn0II0dBIqDUSb731Fo8//jh/+tOfAGjTpg2zZs1i2rRpvPrqq1y+fJmioiLGjRtHdHQ0AHFxcY7t/fz80Ol09XrTVSGEqG8Sao3E4cOHOXDgAPPnz3fMs9lsFBQUkJKSQrdu3Rg6dCgDBw5k2LBhDB06lHHjxhEaGurGUgshRN2SUGskbDYbzz//POPHjy+1LDQ0FK1Wy5o1a9i3bx+bN2/m008/Zfbs2Xz99dd069bN9QUWQoh6IKFWQslrXCaTCW9vb05nFZFddL3VY5sAHUYvz2ln06NHD06ePEnbtm3LXUdRFOLj44mPj+f555/npptuYs2aNXTr1g2DwYDVKvfVEUI0bBJqVWTQKnBDqHlaB+znnnuOyZMnExUVxYQJE9DpdPz222/s37+fV155hX379rF161aGDx9OWFgYR44c4eLFi3Ts2BGA6Ohozp8/z6FDh4iKisLf3x8vLy83vyshhKgez6lqeDhP6YBdnuHDh7Ny5Up27tzJ8OHDGT58OP/6179o1aoVYG/NuGfPHiZPnkyfPn146aWXePbZZ5k8eTIAY8eO5fe//z3jxo2jXbt2rFq1yp1vRwghakTJzMz0rG9nF8nKynL09arItdOPmYU2zuZYHPMD9Artm+nrs4gNSlWPZ3kSEhKIjY2twxKJsshxdg05zu7jtprahx9+yMCBA4mKiiIqKorf//73bNq0qcJtjh07xujRo4mMjKRz587MmzfPZSPme2mdp81y+UkIITyO266ptWjRgtmzZ9OuXTtsNhufffYZ9957L1u3bqVr166l1s/OzmbChAkMHDiQzZs3k5CQwPTp0/H19eXxxx+v9/IaNKVPP6qqiqIo5WwhhBDC1dwWarfffrvT9D/+8Q/+85//sG/fvjJD7YsvvqCgoIDFixfj4+NDXFwcJ0+eZNGiRTz22GP1Hi5ajYJOAUtxxVDFPlxWyRqcEEII9/GIhiJWq5XVq1eTl5dHfHx8mevs3buXAQMG4OPj45g3fPhwLl++TFJSkkvKWaqxiIe1gBRCiKbOrU36jx07xsiRIzGZTPj5+bFs2TK6dOlS5rqpqam0aNHCaV5YWJhjWevWrct9nYSEhFLzvL29q9xk3WQyAaBFAa4HW15hEXoPawXpLtnZ2aSmptZqH2X9nkTdk+PsGnKc609FjXDcGmqxsbHs2LGD7Oxs1q5dyyOPPMKGDRucxiSsq9cpKSsrCy8vr0pPW15r/Qjga7WSbbneQsSm0eLtLV39VFUlMDCQqKioGu9DWou5hhxn15Dj7D5uPf1oMBho27YtPXv25OWXX6Zbt24sWrSozHXDw8NJS0tzmndtOjw8vNqv7efnR2ZmZrVaTxpKXD8rlBaQqKpKZmYmfn5+7i6KEEJ41ogiNpvN6f5fN4qPj2fWrFlONactW7bQvHlzYmJiqv1aOp2OgIAAsrOzK1wvOzvbcRuWqwVW9l0qdCwL8tIQ3EpuzBkQEIBO51EfJSFEE+W2b6JZs2YxcuRIWrZsSW5uLqtWrWLnzp2sXLkSgNmzZ7N//37WrVsHwMSJE5k3bx6PPvoozzzzDKdOneLdd9/lueeeq3HLR51OV2mH4dTUVMdptTydlecOJzuWGQ0qk7rUvMOxEEKIuuW2UEtJSWHq1KmkpqYSGBhIly5dWLVqFcOHDwcgOTmZs2fPOtZv1qwZa9as4ZlnnmHYsGEYjUamT5/OY4895rIyR/pq8NJeP+2YaVbJLLR51MDGQgjRlLkt1BYvXlzt5V26dOHbb7+tryJVSqMoxPjrOJl1fbisxBwLPb0MbiuTEEKI66SKUU1tApxbiyTmSGsRIYTwFBJq1RQT4Fy5TbxhkGMhhBDuJaFWTa1LhNpZCTUhhPAYEmrVJKcfhRDCc0moVVPJmpqcfhRCCM8hoVZNMSVqahfyrBTJ+I9CCOERJNSqyVenIcLn+mGzqnAhV05BCiGEJ5BQq4E2cgpSCCE8koRaDZQ8BXlWGosIIYRHkFCrAWksIoQQnklCrQbk9KMQQngmCbUaaC2nH4UQwiNJqNVAydOPSTmWat1sVAghRP2QUKuBCB8NPtrr93DLLlK5WmhzY4mEEEKAhFqNKIoipyCFEMIDSajVkIzWL4QQnkdCrYZkYGMhhPA8Emo1JH3VhBDC80io1ZDcV00IITyP20LtnXfeYdiwYURFRdGuXTsmT57Mr7/+WuE2SUlJGI3GUo/vv//eRaW+rmRDETn9KIQQ7qerfJX6sXPnTh566CF69+6Nqqq88cYbjB8/nj179hAUFFThtqtXr6Zr166O6crWrw8x/s6H7mKeFbNVxXBDU38hhBCu5bZQ+/LLL52mP/jgA6Kjo9m9eze33XZbhdsGBwcTERFRn8WrlLdOoYWvhkv59v5pKnAu10L7Znq3lksIIZoyj7mmlpubi81mw2g0VrrufffdR/v27Rk1ahRr166t/8KVo3SzfjkFKYQQ7qRkZmZ6xPhODz74IKdPn2br1q1otdoy10lPT2fFihXcdNNN6HQ6vvnmG95++20WL17M5MmTy913QkJCvZR59kkDG1KvB9tz7cxMai4NRoQQoj7FxsaWu8xtpx9vNGPGDHbv3s3GjRvLDTSAkJAQHn/8ccd0r169yMjIYP78+RWGWkUHoDIJCQnlbt89L5sNqTmO6TzvYGJjm9X4tZqyio6zqDtynF1DjrP7uP3049///ndWr17NunXraN26dbW379OnD2fOnKn7glWB9FUTQgjP4taa2vPPP8+aNWtYv349HTp0qNE+jh496rZGI3JfNSGE8CxuC7VnnnmGzz//nGXLlmE0GklJSQHAz88Pf39/AGbPns3+/ftZt24dACtWrECv19O9e3c0Gg0bN25kyZIlzJo1yy3voay+aqqqoijSrF8IIdzBbaG2ZMkSAMaNG+c0//nnn+fvf/87AMnJyZw9e9Zp+VtvvcX58+fRarW0a9eOhQsXVng9rT6Femvw0ynkWextbfIsKldMNsJ8yr8uKIQQov64LdQyMzMrXWfx4sVO0/fccw/33HNPPZWo+q7dgubY1eunHRNzrBJqQgjhJm5vKNLQyRiQQgjhOSTUaklaQAohhOeQUKslua+aEEJ4Dgm1WpKamhBCeA4JtVoq3axfQk0IIdxFQq2Wovx13Ngr7VK+DZPFI4bTFEKIJkdCrZa8tAot/Zxra+dypbYmhBDuIKFWB0qegjwrjUWEEMItJNTqgDQWEUIIzyChVgdkYGMhhPAMEmp1QE4/CiGEZ5BQqwMlTz8mSU1NCCHcQkKtDpQ1qoiqSrN+IYRwNQm1OhDkpSFQf723WoFVJaXA5sYSCSFE0yShVgcURSFGGosIIYTbVTvUTpw4wddff+0078cff+TOO+9k+PDhLFq0qM4K15DIwMZCCOF+1b5J6EsvvYSiKNx+++0AXLx4kcmTJ+Pl5UVYWBgvvfQSRqPRo27m6QpyXzUhhHC/atfUDh8+zKBBgxzTn3/+OTabjZ07d7J7925GjRrFkiVL6rSQDYF0wBZCCPerdqhlZWUREhLimP7f//7HzTffTPPmzQEYNWoUp06dqrsSNhAlTz8myelHIYRwuWqHWlhYGOfOnQMgMzOTn3/+mWHDhjmWFxYWVmk/77zzDsOGDSMqKop27doxefJkfv3110q3O3bsGKNHjyYyMpLOnTszb948j2g+LzU1IYRwv2pfUxs2bBj/7//9PwIDA9m5cycAo0ePdiw/fvw4LVu2rHQ/O3fu5KGHHqJ3796oqsobb7zB+PHj2bNnD0FBQWVuk52dzYQJExg4cCCbN28mISGB6dOn4+vry+OPP17dt1KnWvlr0ShgK87X5AIb+RYbvjppYCqEEK5S7VCbOXMmp06d4h//+AcGg4FXXnmF6OhoAEwmE1999RV//OMfK93Pl19+6TT9wQcfEB0dze7du7ntttvK3OaLL76goKCAxYsX4+PjQ1xcHCdPnmTRokU89thjKIpS5nauoNcotPLTci73+mnHpBwrnYMk1IQQwlWqHWphYWF8++23ZGVl4ePjg8FgcCxTVZV169bRqlWrahckNzcXm82G0Wgsd529e/cyYMAAfHx8HPOGDx/O66+/TlJSEq1bt67269alNgE6p1BLzLHQOUjvxhIJIUTTUu1Qu6ZZs2ZO06qqoqoq3bp1q9H+XnjhBbp160Z8fHy566SmptKiRQuneWFhYY5l5YVaQkJCjcpU3e2DbAZuPKR7zyTTvlCurVVVbX9PomrkOLuGHOf6ExsbW+6yaofahg0bOHDgADNnznTMW7BgAXPmzMFkMnHrrbeyZMkSfH19q7zPGTNmsHv3bjZu3IhWq618g2qq6ABUJiEhocrb9yjI4auUbMd0nncQsbHGGr92U1Kd4yxqTo6za8hxdp9qX/B59913SU5OdkwfOnSIl19+mT59+vDggw/yv//9j/nz51d5f3//+99ZvXo169atq/T0YXh4OGlpaU7zrk2Hh4dX/U3Uk5L3VZPR+oUQwrWqHWqnT5+me/fujukvvviC4OBgVq1axTvvvMOUKVNKNQIpz/PPP+8ItA4dOlS6fnx8PLt27cJkMjnmbdmyhebNmxMTE1Pdt1Ln5L5qQgjhXtUONZPJ5HRqcfPmzQwfPhwvLy8AunXrxsWLFyvdzzPPPMOKFSv48MMPMRqNpKSkkJKSQm5urmOd2bNnM3bsWMf0xIkT8fHx4dFHH+XXX39l3bp1vPvuuzz66KNubfl4Tan7quVasHlAHzohhGgqqh1qLVu25ODBg4C91nb8+HFuueUWx/KMjAy8vb0r3c+SJUvIyclh3LhxdOzY0fFYsGCBY53k5GTOnj3rmG7WrBlr1qzh8uXLDBs2jGeffZbp06fz2GOPVfdt1Aujlwaj4Xq4FlohOV9uQSOEEK5S7YYikydPZs6cOVy+fJnjx48TFBTErbfe6lh+4MAB2rdvX+l+MjMzK11n8eLFpeZ16dKFb7/9tlpldqXWAToOpRc5ps/mWGjhV/eNX4QQQpRW7ZraU089xVNPPcWlS5do1aoVy5YtczTvv3r1Kj/99FO5naebAhkuSwgh3KfaNTWtVstLL73ESy+9VGpZUFBQk++bIfdVE0II96lx52uAK1euOAY3jo6OJjQ0tE4K1ZBJTU0IIdynRqG2a9cuXnzxRQ4dOuQ0v3fv3rz22mvcdNNNdVG2Bqlks34JNSGEcJ1qh9quXbsYP348/v7+TJ8+3dG/7OTJk/z3v/9l3LhxrF27tskGW0ypmpqcfhRCCFepdqi9/vrrREdHs2nTJoKDg52WPfXUU4wcOZLXX3+d9evX11khG5JWflp0CliKu6elmWzkFtnw18to/UIIUd+q/U178OBB7r///lKBBvaGIvfff7+jH1tTpNMoRPlLYxEhhHCHaoeaVqvFbDaXu7ywsBCNpmnXSqSxiBBCuEe106d///4sWbKExMTEUssSExNZsmQJAwYMqIuyNVglBzaWUBNCCNeo9jW1l19+mdtuu43+/ftz2223OUYPSUhIYOPGjXh5eTndlqYpKt0CUk4/CiGEK1Q71Lp27coPP/zAK6+8wv/+9z/Wrl0LgK+vL6NGjWL69OmOwY2bqtItIKWmJoQQrlCjfmodOnRg2bJl2Gw2rly5AkBoaCgajYa33nqLN954g4yMjDotaEMio4oIIYR71GpEEY1G4xE35/Q0JWtqSbkWrDYVrcb9t8cRQojGrGk3U6wnzQwagr2uH9oiG1zKl9qaEELUNwm1eiKnIIUQwvUk1OpJyb5qZ6WxiBBC1LsqXVPbv39/lXd46dKlGhemMSnZrD9JQk0IIepdlUJtxIgRKErVGjmoqlrldRuz0qOKyOlHIYSob1UKtffee6++y9HoyFBZQgjhelUKtXvuuadeXvzHH39kwYIFHD58mMuXL/Pee+9x7733lrt+UlISPXr0KDV/1apVjBgxol7KWFMlTz+elZqaEELUu1r1U6utvLw84uLiuPvuu3n44YervN3q1avp2rWrYzooKKg+ilcrLXy16DX25vwAGYU2ssw2mhmkbY4QQtQXt4bayJEjGTlyJACPPvpolbcLDg4mIiKivopVJ7QahRh/Haeyr592TMqx0D3E4MZSCSFE49Ygqw333Xcf7du3Z9SoUY6xJz2RnIIUQgjXcmtNrbr8/f159dVXuemmm9DpdHzzzTdMmTKFxYsXM3ny5HK3S0hIqNXr1nR7o00P6B3T+88mE1ckDUbKU9vfk6gaOc6uIce5/sTGxpa7rEGFWkhICI8//rhjulevXmRkZDB//vwKQ62iA1CZhISEGm/fszCHVZezHdO5XkHExhprXJbGrDbHWVSdHGfXkOPsPg3y9OON+vTpw5kzZ9xdjDLJqCJCCOFaDT7Ujh496rGNRkqG2ulsC6qquqk0QgjR+Ln19GNubq6jlmWz2bhw4QJHjhwhKCiIqKgoZs+ezf79+1m3bh0AK1asQK/X0717dzQaDRs3bmTJkiXMmjXLje+ifCUbipzLtfK3nzJ5e4BRbkMjhBD1wK2hdvDgQcaMGeOYnjNnDnPmzOHuu+9m8eLFJCcnc/bsWadt3nrrLc6fP49Wq6Vdu3YsXLiwwutp7uSv1zAo0sCPyWbHvP87mU9Okcr7g4PQS7AJIUSdUjIzM+V8WAVqe8H316tFjPn2CumFNqf5o6K8+b+hwfjoJNhALqy7ihxn15Dj7D4N/pqap4sL0vPN6FBa+Dof6k3nTfzxf1fIKbKVs6UQQojqklBzgY5GPd+ODit149AdyWbGb7zC1UIJNiGEqAsSai4SE6Dj29FhdDY6X8bcf6WI279JIzlfRhsRQojaklBzoUhfLV/fFkrvUL3T/F8zLdz2TZrcSFQIIWpJQs3Fgr21fDUqlEGRzgMbn82xMvqbK5zMLHJTyYQQouGTUHODQIOGVb8PZVQrL6f5F/OtjP72CofTzeVsKYQQoiISam7io1P49JYQ7mzj4zT/isnGmI1X2JNS6KaSCSFEwyWh5kYGrcKHg4O4v4Ov0/xss8qE79LZctHkppIJIUTDJKFW33KzMHz5EYZVSyAvp9RirUZh/kAjj3Xxd5qfb1GZ/H0665MKXFVSIYRo8CTU6pGSkYrv7EcwrP0Yw/pl+PzzabCWbuGoKAqv9gtkRq8Ap/lmGzy4JYPPTuW7qshCCNGgSajVEyUjFZ85T6JJveSYp008iX7zurLXVxSe6xnInPhmTvOtKjyy4yof/pZbr+UVQojGQEKtHigZafjM/ZtToF1jWPMR5GaVu+0jXfxZMMhIybGOn92dxTtHSp++FEIIcZ2EWh1zBFrKxbKX5+XgtXpphfu4r4MfS4cEoy/x23llfzazfs6Se7IJIUQ5JNTqkHL1SnGgXXCabwt1vompbst6NOdOVbiv8W18+Gx4CD5a5yrbu0dzeXjHVfItMl6kEEKUJKFWR8oLNEvv35H/+kfYIlpeX1e14bV8AVRS4xrRypvVI0MI0DsH2+enC/j9hjTOZMuwWkIIcSMJtTqgZKbbAy35vNN8S+9BmKa/DN6+FN493WmZ9vhhtPu2VbrvgZFerLs1lGAv51/VsasWhq5LZYM0+RdCCAcJtVqyB9qTpQOt1yBM02eBzj54sbXnACzd+jmt4/XfxVBYeQfrXqEGNo4OpUMz5xH+s4tU/rQ5g5f3ZWGxyXU2IYSQUKsFRw3tcolA6zkQ02OzHIFmX1mh8J7HULXX76mmSU9B/81/q/RaHYx6fhgTxoTWPqWWzf8ll/GbrpAit68RQjRxEmo1pGRl4DPvKTSXzznNt/QcUDrQiqktYigacafTPMM3n6Gkp1TpNQP0GpYODWJu/2boSjT535lsZsi6VHbJmJFCiCbMraH2448/ctddd9G5c2eMRiPLly+vdJtjx44xevRoIiMj6dy5M/PmzXN5E3clKwPvuU+huZTkNN/S4yZMj80GvaGcLcE87n5sAcbr+zIXYvjv+1V/bUXh4Th/vr4tlOa+zr++5AIbd3x7hfeO5UqzfyFEk+TWUMvLyyMuLo65c+fi41P6tFpJ2dnZTJgwgfDwcDZv3szcuXNZsGABCxcudEFp7ZTsq3jPfQrtpUSn+ZYeN2F6/JUKAw0AvwDME//iNEu/dwua44eqVY7+EV5sHxvO4ObOt6+xqvDi3iwe3JpBtlma/Qshmha3htrIkSOZOXMm48aNQ6OpvChffPEFBQUFLF68mLi4OMaNG8df//pXFi1a5JKaiT3Q/lY60Lr3r7SG5rT+4NuwxnRwmue1fAHYqndNLMxHy5qRITzd3b/UsrWJJm5Zn8avV+Wmo0KIpqNBXVPbu3cvAwYMcKrVDR8+nMuXL5OUlFTBlnUgOxPveU+hvZjoNNvSLd5eQzN4lb1dWTRaCv/0uNMs7bnT6LZuqHaxtBqFf/RpxmfDgwk0OF9oO5VtYcSGNFaelgGRGyRLEZqkBHQ/fodu6wZ0P36Hdu9WtAd/QvvLz2iOH0Zz+jc0506jJJ9HuZKMkpUB+blgLqy0H6QQjZGu8lU8R2pqKi1atHCaFxYW5ljWunXrMrdLSEio1euePXSA9svfRpvqPPRVdruunLn9QdSkc+VsWQHFm5gu8QQf2+uYpVv5ISdDY7D6+FV7d+2Bj7spPH/ci5N51/9XybeoTN1+le8SUvlbmyIMHvxvTG1/Tw2ZptCET8p5fFLO45t8Dp+U83inXUJTxl0dqsOm1WHT6VF1OlStnthmIWS3jSOnXRfym8eA4sEfiAauKX+e61tsbGy5yxpUqNVURQegMmcOHyDui4WlAs3StR+av75G++rU0EpQ/vIM6vP3o5jtfdV0Bbl0OrId85+eqNH+YoFtcSrP7s5kWYJz7WzVZT1ni3z5v2HBRPl73q89ISGhVr+nhkTJvoomKaH4cQptUgJK6kWUeqhZaawWezAWN4o1ZGfgfz4Btq1FDWiGpWs/rN3isXbti9osuM5fv6lqSp9nT+N5324VCA8PJy0tzWnetenw8PC6f8GcTGKXvYM2tcTQV136Yvrra9U75VgGNTgc85h78Vr9H8c8/Q9fYRl6B7ZWbWu0Tx+dwsLfBREfbuDZ3ZkU3nCZbv+VIoasS2PJkCBuaeldq7KLKlBVlLTLaJIS0CYloDl3Ck3SKTSZV9xdMgCUnCz0u75Hv+t7AKwxsVi7xWPpFo+tfRfQNaivByGABhZq8fHxzJo1C5PJhLe3/Ut5y5YtNG/enJiYmLp9sdwsfP75dNmB9uTrtQ60a4pu/SP67d+gSbsMgGKzYVi+ENNzb4OiVLJ1+e7v4EePED33b84gKfd6smUU2vjDd+m80CuAZ7oHoC15jxtRO6qKJvEE+u3fotu7BSU3u1a7swWFYouJRQ0MgiIzSpEZLEXXnxfd+Ny+7NpzxVK9RkLa4vA1bFiO6u2LtUsfLN3sNTk1NLJW70MIV3FrqOXm5nLmzBkAbDYbFy5c4MiRIwQFBREVFcXs2bPZv38/69bZb6w5ceJE5s2bx6OPPsozzzzDqVOnePfdd3nuuedQahEAVWXp0qdOAw0AgxeFdz2Kz4J/OGbpfj2Adv9OrH1vrtWue4QY2DY2nGk7rrLp/PXhuFRgzsEc1p4t4O+9A7kj2tslx69Ry8lE/9P/0G3/Fu2FM9XeXFUU1MhWWKNjscXEYotujzUmFgKNNS+TzeYcgKZ8rmz/Hy1Tk9Ae+xklP6/cTRVTPrr9O9Dt32HfVfNoLN3isXaPx9qxR93+DQhRh5TMzEy3NZHasWMHY8aMKTX/7rvvZvHixTzyyCPs3LmTo0ePOpYdO3aMZ555hgMHDmA0GpkyZQrPP/98/Xwp52aheeUxfFPOY4nrjenJN8CrHk7bqSre/3wa3a8HHLNsYc3Jf+P/6uTLw6aq/OtILq8fzKasISJ7hOh5sVcgv2/l5bZwa5DXIGxWtEd/Rr/jG7QHfkSpYqMOVafH1qqNU3jZotqCt289F/iG42y1oDn9G7qje9Ee3Ysm8WSVr+mpegPWDt2wxXbFGtsNa7vOUIPGTY1Zg/w8NxJuDbWG4Mzhg/bGG3+cVj+BVkxz4Sw+/3gIxXa9w3ThHx6iaOx9dfYa2y6ZeGjbVa6Yyu6U3S9Mz0u9Axnc3PXh1pC+BJSUC+h3bES3cyOaqxVfH1N9/LDFtMca3b44xGKxtYhx2/Wqco9zdia6Yz+jPbIX7S/70GRfrfI+VUWDLaoNtvZdsbbvgrVDN/vpyiZc+29In+cqMeWjZKajBhjBL8DdpamQhFolXPnhNCz7N4b/femYVg3e5M/7BDW47hrBXMqz8vLPWaw6U0B5v/jfRRp4sXcgAyJcd4rJ478ECgvQ7duGfvu3aE8crnBVVaPB2v0migbfhrXHAI9qcFGl42yzoTl3Cu3RfeiO7kWTcNTpn62qsDULLq7J2YPO1rpDmeOhNlYe/3kuqSAfTXoyypVkNGnFP68U/0xPQcnJcqxqC2uOtXVHbG06YGvdEWvrDh4VdBJqlXDphzM3G7/n/+TUuKBowAgKH36pzl/qt6tFzD2UzdrE8m99M7ylFy/2CqR3WNVGSqkNj/wSUFU0p39Fv/0bdHs2o5gqvnedLTKKosGjsQwaiWoMcVEhq6dGxzk/F+2vB6+fqqziANw3UvV6bG06FYecPeiqfL1QVcFqBcu1hjBF168VWorAYkH1D0QNjQCNtvL9uYDHfZ4L8q+HVFk/a9ugKbyFPehad8DWpqP9erCbgk5CrRKu/nDqfliL9yf/cpqX/+ICbB261cvrHU4388bBHKeGJCXdFuXNjN6BdAuuv/+0PelLQMnKQPfjd/ZWqZcr7livenlj6X8LRTffhi22q8efcqv1cVZVlJQLaE/+gvbUL2gTfik1sHdV2SKjsAWFFofUjS037aGlWG54XoXrfapejy0iClvzaNQWMdiaR2NrEY0tMqpeLx2UxRM+z0ryefTfrUb38zY0WVU/nVxXbBEtsba21+YcQedbeki/uiahVgmXfzhtVnxmTkV7/rRjljWmAwWzFtfrf6E/p5l5/UA2Wy6Vf+uaCa19eKFXAB2NdR9ubvsSyM2y9yFLTECTdBJtYgKalAuVbmbt0I2im0djiR/ikgYedaVejnNuNtpTx9CeOoYm4Re0Z35DMXvWLZBsoRH2kGseja048NTm0fauEvXwj4jbPs+qivbXA+g3fYHu8O66261Wi9osGCUzvdqnom9ki2iFtXUHCv+/F6o8Vm51ec7JfmFXPC6k75wnHbO0SSfRbf8Wy9A76u1l+4YZWDMqlB+TC3ntQDa7Usyl1lmTWMDapAImtfXhhZ6BtAlsWB8fJTPdPopH4kl7kCWdRHOl6qfSbMYQLINGUTT4NtTIqHosaQPjH4i15wCsPQfYpy0WNOdPoU0oDrmEo5U2qKlvmisp9t/10X1O81W/gOth1zwaW+sOWDt2b3jX/8yF6Hb/gH7Tqpp1KdHqUEPCsYVGooZGlvqpBoXY/6k2F9qvtyaeRHP2BJrEk2guJqKoVQs6TcoFKDTVW6CB1NQq5a7/uLwWzkK/b6tj2hZgJH/epy45T62qKlsvFfL6wWx+TivdgTfAks/AnFP8RX+OwYFmAgzFNUhFsY8lqAAoxdM3PFBAsbeWc/x3XLzsSmYWIa2iwdsX1ccH1dsXvHxQfXzt87x9wOANVbibA6qKkpF2veaVeMIeZpnp1T8WWi3WngMpGjwaa7d+oG1YQV6Suz7PSnqK/VTlqWNoT/6C5vypav3Hr2o0oDOAXo+q04NeD3oDqs4AGi1KRiqanMw6Kavq7Yu1Wz8sPQdg7d7fXpurJlcdZyUzHf3mdeg2r63w/ZcbWmHFoWUMqfmZoEITmvOn0Z49Yf9bSzyJ5mJSuUFn6TXI3t+3njTsv9BGzHzXw+gO/WS/xgBocjIxrP0E8z3T6/21FUVhWEtvhrbwYtP5Apb+eJbgpGMMzDrJwKyTdMs7j7bctpM107IK66iKAl7e9sDz9kX18oHiALSHoDdKeirapJNOrbWqS1UUbFHtsAwaiWXg72v0pSacqSERWEIi4Kbh9hmmfDTnTqEUmVH1Bntg6fSoer29lqQ3FIeXPciq9IWbm4Xm8nk0l5LQXD5nf1xKQklLrnJNAoo7nu/bhm7fNvtnoW1nLL0GYu0xwN6f0AOum2qSEtB/twrd7s0VjhxjbdORolGTsPQbUn+1Ty9vbO272IdWu6awAM250zfU6E6guXQORbXZW0vWI6mpVcKdF3wNaz7C8NXHjmlVqyX/taWoLep4SLAbWS32D2PCLx5z6qg+qVottpatscV0wBYTa7+w7aKO0O7gCQ0YXM5ciCb1IsqlczeEnf3ntcHEq8oWEmGvwfUYgLVzz3IHR6iX42yzoj20C/2mVegquKmwqmiw9r0Z88iJntV4qbAATdIp1KBQ1LDm9fYyEmqVcOuXQKEJ3xfuR5OR6phl6dYP09P/rLsPan4u2tO/Xg+x07+iFFbvD72hUPV6bK3aYWsdizWmA7bWsdhatmlSQz41yVArj82GcjWtOOCS7HdMOLqnyi0FVYM31q59sPSwX0+8sRtHnR7ngnz0O75F/7/VaFIvlV8eHz+KhtxO0YgJ9Roank5CrRLu/hLQ7dmC96LZTvOsHXugennbr+9otahabfFz+8MxrdPdsM715Wg09v9WE35Bc+FMjW55khEaxUaf9hzS2/94FNTiBxg0MDRST/8wA1qleN82GwqqfeBJVPu4hGDvg6TayExLJcjbC0wFKKZ8FFO+47ljXjVa1Kle3vYhqFrba2C2mA5uHcnDU7j78+zxbDY0iSfRHdqF9tBPaJOqfk80a5uOjoA7abLSvl27WhVFyUxH/8NX6Ld9jVJQ/jidtvAWFI2cSNHvbgWfxnmGoTok1Crh9i8BVcVn7pNoj1c8ikW9FkFvuN5xtoO98yz+gWSbbcw9lM0Hv+ZhLeNT1Mmo460BRn4XWXlNqErH2WqBQtP1oCvIRykssA/hU2Cfh68/1pj2qJGtPKYjridx++e5gVEy0tAe3m0PuV/3e1RXBUunnhSNmmhvdSqfdQcJtUp4wpeA5twpfGZOrdbF7tqwBQZdH+IotmulQxwdzSji6Z8y2ZtWuhsAwOR2PrzWrxlhPuX/4XnCcW4K5DjXQqEJ7W8H7QF3eBeajLTKt6ljqk6P5aZbKBo5EVuM/B7LIqFWCU/5EtBvWI7XFx/Wy76tLVrbQ6yDPcTU8JbVvmZnU1WWJeTz8s9ZXC0s/ZFqZlCY2SeQBzv4lXkPN085zo2dHOc6oqr2/lqHdqE7vAvNmeP1cufya2wBRiy3jKPolrEeOwSbp5BQq4QnfQko6aloUi+CxWI/FWezgtWCYrH/vPZQrDdMW0pMWy32cfT8ArC2i7OPwecfWGdlTDdZefnnbJYl5Je5vHeonncGGOkZ6tz50pOOc2Mmx7l+KFkZaA/vQXd4F9rfDqGa8lEqvQFvJcsVBVvLNhTdMhbLTcObVIOm2pBQq4R8CdTM7pRCntqVya9XS99jTKPAQ538eLFXIEYve2dqOc6uIcfZNeQ4u08VhmcQovpuivBi29hwXusXiL/O+T9Smwof/pZH/JoUVp7OR63H0zZCiKZFQk3UG71G4bGuAey5M4JxrUuPkp5aYGPq9quM3XiFk7kKhWU1oRRCiGpo2p12hEu09NPy8bAQvr9g4tndmZzNsTot35FsZkeyDxy6hLcWjAYNRi8NzQwajAaFZo7nGpoZFIxe155rip8rNDNoCNArLr9jtxDCs0ioCZcZ0cqbn8ZH8O7RHP51JAdzGT0UTFZILrCRXFD97gsaBcK9NUT764gJ0BLjryO6+GdMgJaWflr0lV68F0I0ZG4PtSVLlvDvf/+blJQUOnXqxJw5cxg4cGCZ6+7YsYMxY8aUmr937146dKjfQTJF3fDRKfy9VyB/bOvLs7sz2VzB/duqy6ZeC0Qze8voQqRVoIWflhh/LTEBOsfPaH978EX6atBITU+IBs2tofbll1/ywgsv8Pbbb3PTTTexZMkSJk2axO7du4mKKv9+Vbt37yYo6PrI6aGhoa4orqhD7ZrpWD0yhLWJJt7/NZcTVwvJsShY6vGymlWF87lWzuda2ZlcuqO4lxai/K7X6sK8NYR6awnz0RDm+Kkh2EtTZl87IYT7uTXU3nvvPe655x4eeOABAN58801++OEHli5dyssvv1zudmFhYYSESAfEhk5RFMa38WF8Gx8SEhJo3749eRaVLLNKZqGNTLONLLOt+Ll6w3MbWWVM59cyEQutcCrbwqns0t0QnMoNhHhrCPfWEOpzLfw0hPvYgy/U+3oIRvpo8dZJAArhKm4LNbPZzKFDh3j88ced5t9yyy3s2bOnwm2HDh2K2WymY8eOPPPMMwwePLg+iypcRFEU/PUK/np745LqKrSqXMqzkpRr4VyulaQcC0k3/EytwXW6sqjAFZONKyYbZFYegK38tbQP1NHuhkf7ZvbTnjqp8QlRp9wWaunp6VitVsLCwpzmh4WFkZqaWuY2kZGRvPPOO/Tu3Ruz2cznn3/OuHHj+Prrr8u9Dgf2jpC1UdvtRdXU1XFuCbRUYEAgcMNgKSYrXC5UuGRSuGjScKlQ4bJJ4VLx82xL3QeMyvVTnltKXD/UKiotvVWivVWifWxE+9h/RvmohBtU6ivvPOHzbLLC6XwNvlqV1j6qx9zyqy55wnFurCrq2O72hiLVERsb6/Rm4uPjOXfuHP/+978rDLXa9OyXkQFcw1XHuVsFy7LMNpJy7LW81AIbqQVWrphspJmspBXYSDPZSCuwkmmumwt/VlXhXIHCuQLgqnPN1Eer0Cbweg2vhZ+WQIOGQL1i/1n8/FpXhqpe43PX59lkUdmbZmZnciE7LheyP83saP0a7qNhcHMvhjT3YkgLL6L9G9TXUpnke8N93PbpCQkJQavVkpbm3EwtLS2N8PDwKu+nT58+fPnll3VdPNEENTNo6B5ioHsll2vNVpX0QnvA2YPOHnxXbgi+NJM9FC/n26hJBBZYVX69ailzmLGy+OsUAg3FgafX3PD8egg2MyjYMrUUhRbRJkCHTz1e6yu0qvycZmbH5UJ2JheyL81MobXsdVMLbKw6U8CqMwUAtAnQMrSFF0Oae3NzcwMh3nJbFVF1bgs1g8FAz5492bJlC+PHj3fM37JlC2PHjq3yfo4ePUpEREQ9lFCIshm0Cs19tTT3rfzL1mRROZtjb3xyJtvCqSwLp7Ptj5Q6usYHkGtRybWoXMqvbJ9ecDwVBft1y/bNdE7X+2p6rc9sVTlwxR5iO5LN7E0txFROiFXmbI6Vsyfy+ehEPgrQLVjPkBb2mtyACAN+ehkISZTPrfX86dOnM23aNPr06UP//v1ZunQpycnJTJkyBYBp06YB8MEHHwCwaNEioqOj6dy5M2azmZUrV/L111/zySefuO09CFERb51C5yA9nYNK348u22zjTHHAnSr+eTrL/jyrjk5xlkcFLuRZuZBnZWuJa306BdrcGHSBOtoVh19zXw2KolBkUzl4xczOZHuQ7Uk1V7v1aZS/lqsmG7kVbKcCRzKKOJJRxIJfctFroF+YgSEtvBja3IveYQbpUC+cuDXU7rzzTjIyMnjzzTdJSUmhc+fOrFy5kujoaAAuXLjgtH5RUREzZ87k0qVLeHt7O9YfOXKkO4ovRK0EGjT0DDWUug2PqqpkFNoctboz2VbSC61km1WyzTayi4p/mlWyi2zkFNVtAFpUSMiykJBV+tSnr06htb+Wc7nWCsOoLFH+Wm6O9OJ3kQZubu5FlL+OIpvKgTQzWy8Xsu2S/TRlUQWVzSIb/JRi5qcUM3MO5uCvUxgUaaBfuBcx/lpaB9j7GYZ5a2TItCZKbj1TCbng6xpynGvOalPJKbL327seeCXCr7jP3/HUHC5ZDJzPtdboWl91tPTV8rvm9gD7XaQXrQMq/x86r8jGrhQz2y4XsvVSIUczimr02r46xT5SzA0jx9z4M9BQv6cw5fPsPg2/mZEQTZxWo2D0Uhz3pqtIQsIVYmOjy7zWd+0UaE3780X6aLi5uZf9EelF6wBttWtLfnoNI1p5M6KV/a4O6SYrOy6b2XrJxLbLhaUGwy5PvkXleKaF4+X0Iwz20jjGB72xhhfpqyXIS0OQQSOd5hsoCTUhmqCqXOs7VaJhy6ksC9k3nOoM99EUn0704ubmBtoF6ur8lF+It9Yx6gxAUo6FbZcL2V58ujLNVLMAzii0kVFo4+CV8muCPlqFoOJ/Fq4FXZBX6YfRoCHIS3FM+0kYupWEmhDCSUXX+q6YbCTlWjEalHoJscrEBOi4P0DH/R38UFV7t4ddKYWcybGQmGN19DOsi+uMBVaVgvyqtCh1ptdACy9vBiZfJT7MQN9wA3FGnYwX6iISakKIKlEUhTAfLWE+ntFvTFEUugTr6RLsXNtUVZWrhfbwTcyxkJRjHzrt2s9zudYKG6PUVpENkgo0JJ3K57NT+YC9H2HvMAP9wvT0CzfQN8xAqPS/qxcSakKIRkVRFIK9tQR7a+lVorYJ9oY1l/OtTuOCJhbX8DJMNq6abVwttNVp8OVaVLYXnza9pm2Alr7hBnttLsxA12C9jAVaByTUhBBNilaj0MpfRyt/HYMivcpcR1VV8iz2Gp/9oZJptt0wfcOjeH5m8XoF1qqd+jyTY+VMTgErT9tHUvHVKfQK1dMvzGB/hBsI95BacUMioSaEECVcv2OEhij/6m2bbbbx9eGzXPYKZ2+qmX2pZtILK6/25VtUfkw28+MN9/oL9tIQ6aMh3FdLRPGtjCJ8tY55kT4aIny1BMgoKw4SakIIUYcCDRr6Gm3ExgYA9lpfYo6VvWlmfk41szfNzC8ZRVSlQnetleavldziyE+nEFEccBE+xQFYHIQRvlpa+WmJ9q/f8T49hYSaEELUI0VRaBOoo02gjsntfAHIt9i7E/ycZrbX5tLMtbrfX55FLT6dWXE/vkgfDa0DdEQH2PvmtS7ukN46wD4EmqYRjMIioSaEEC7mq9MwKNLLcU1PVVXO5Vr5Oc0ecPtSzRzJKKrzVprJBTaSC8zsLuOWlQYNRPvbO6FfC7zoAB2tizupV6VzvyeQUBNCCDdTFMU+hFeAjj+0tdfmLDZ7v8DkfCspBTZSCqykFD+3z7s+v7zb+lSH2Ya9w322BSgstdxoUOho1NMjRE/PED09Qw10aKbzuBabEmpCCOGBdBqFSF/70F0VUVWVLLNK8g2hl5JvJbk48C7nWzmXa+VinhVbLfqkZ5pV9qSa2ZN6vSGLj1ahW7CeHqH2oOsRYqCT0b1BJ6EmhBANmKJcH/uzk7H0sGfXFNlULlzrkH5Dx/TEXAuJORauFlY/8Qqs9jua7027HnTeWugarKdniMFeqwu1B52rbhEkoSaEEE2AXnO9wUpZssw2p87oScXDjiUWd1I3V/H6nskKP6cV8XPa9XE1vbTQNcgecD1C9NzVzheDtn5CTkJNCCEEzQwauocY6B5SeplNVbmQZ+VIehGH0os4fMXMofSiKg8oXWiF/VeK2H+liEC9wp9ifeu49NdJqAkhhKiQRlGI9tcR7a/jjhj7HRNU1T7Y86HigDuSbv+ZUknXhO4h+nrtOiChJoQQotoURaGln5aWfj7cXhx0AJfzrY6gu1arS74h6HqGlB6Psy5JqAkhhKgzzX21NI/24bbo60GXnG/lcHoRh9LN/K6c8TbrioSaEEKIenWta8KoKO96fy23dxFfsmQJ3bt3JyIigiFDhvDTTz9VuP7OnTsZMmQIERER9OjRg6VLl7qopEIIITydW0Ptyy+/5IUXXuDpp59m+/btxMfHM2nSJM6fP1/m+omJifzxj38kPj6e7du389RTT/Hcc8+xdu1aF5dcCCGEJ3JrqL333nvcc889PPDAA3Ts2JE333yTiIiIcmtfH330EZGRkbz55pt07NiRBx54gLvvvpuFCxe6uORCCCE8kdtCzWw2c+jQIW655Ran+bfccgt79uwpc5u9e/eWWn/48OEcPHiQoqKiMrcRQgjRdLitoUh6ejpWq5WwsDCn+WFhYaSmljGENJCamsrQoUNLrW+xWEhPTycyMrLM7RISEmpV1tpuL6pGjrNryHF2DTnO9Sc2NrbcZU2i9WNFB6AyCQkJtdpeVI0cZ9eQ4+wacpzdx22nH0NCQtBqtaSlpTnNT0tLIzw8vMxtwsPDy1xfp9MRElLG2C51QD6YriHH2TXkOLuGHGf3cVuoGQwGevbsyZYtW5zmb9myhf79+5e5TXx8fJnr9+rVC72+/NGphRBCNA1ubf04ffp0VqxYwSeffMKJEyd4/vnnSU5OZsqUKQBMmzaNadOmOdafMmUKly9f5oUXXuDEiRN88sknrFixgscee8xdb0EIIYQHces1tTvvvJOMjAzefPNNUlJS6Ny5MytXriQ6OhqACxcuOK3funVrVq5cyYwZM1i6dCmRkZHMmzePcePGuaP4QgghPIySmZlZi3uhCiGEEJ7D7cNkCSGEEHVFQq0C1R2XUlTPnDlzMBqNTo8OHTq4u1gN3o8//shdd91F586dMRqNLF++3Gm5qqrMmTOHTp06ERkZye23385vv/3mptI2TJUd40ceeaTUZ3vEiBFuKm3TIqFWjuqOSylqJjY2lhMnTjge8o9D7eXl5REXF8fcuXPx8fEptXz+/Pm89957zJs3j82bNxMWFsaECRPIyclxQ2kbpsqOMcDQoUOdPttffPGFi0vZNEmolaO641KKmtHpdERERDgeoaGh7i5Sgzdy5EhmzpzJuHHj0Gic/8RVVWXx4sU8+eSTjBs3jri4OBYvXkxubi6rVq1yU4kbnoqO8TVeXl5On+2goCAXl7JpklArQ03GpRQ1k5iYSKdOnejevTt//vOfSUxMdHeRGrWkpCRSUlKcPts+Pj4MHDhQPtt1bNeuXbRv354+ffrwxBNPlBo4QtSPJjFMVnXVZFxKUX19+/Zl0aJFxMbGcuXKFd58801GjhzJ7t27CQ4OdnfxGqWUlBSAMj/bly9fdkeRGqURI0YwZswYYmJiOHfuHK+99hpjx45l69ateHnV752fmzoJNeE2v//9752m+/btS8+ePaVDvWjw/vCHPzied+nShZ49e9KtWzc2bdrE2LFj3Viyxk9OP5ahJuNSitrz9/enU6dOnDlzxt1FabQiIiIA5LPtYs2bN6dFixby2XYBCbUy1GRcSlF7JpOJhIQExxevqHsxMTFEREQ4fbZNJhO7du2Sz3Y9Sk9P5/Lly/LZdgE5/ViO6dOnM23aNPr06UP//v1ZunSp07iUovZeeuklbr31Vlq1auW4ppafn8/dd9/t7qI1aLm5uY4agc1m48KFCxw5coSgoCCioqJ45JFHeOedd4iNjaV9+/a89dZb+Pn5MXHiRDeXvOGo6BgHBQUxd+5cxo4dS0REBOfOneOVV14hLCyMO+64w80lb/xkmKwKLFmyhPnz5zvGpXzjjTcYNGiQu4vVaPz5z3/mp59+Ij09ndDQUPr27cuLL75Ip06d3F20Bm3Hjh2MGTOm1Py7776bxYsXo6oqc+fO5f/+7//IzMykT58+vPXWW8TFxbmhtA1TRcf4nXfe4d577+XIkSNkZWURERHBzTffzIsvvkirVq3cUNqmRUJNCCFEoyHX1IQQQjQaEmpCCCEaDQk1IYQQjYaEmhBCiEZDQk0IIUSjIaEmhBCi0ZBQE0KQlJSE0WjkX//6l7uLIkStSKgJ4SLLly8vdTfkGx/ff/+9u4soRIMnw2QJ4WIvvPACbdq0KTW/a9eubiiNEI2LhJoQLjZ8+HD69evn7mII0SjJ6UchPIzRaORvf/sbX375Jf379yciIoJBgwaVeXoyKSmJKVOm0KZNGyIjIxk2bBgbNmwotZ7ZbObNN9+kX79+hIeHExsby913381vv/1Wat2PP/6Ynj17Eh4ezrBhwzhw4EC9vE8h6oPU1IRwsezsbNLT00vNDwkJcTzfs2cPa9asYdq0afj7+/Pxxx9z1113sX79egYMGADY74E2atQocnNzmTZtGiEhIaxcuZL77ruPDz/80DHqvs1m46677mLz5s2MHz+eqVOnkp+fz44dOzh06BCdO3d2vO6XX35JXl4eU6ZMQVEU5s+fz3333cehQ4fQ6/X1fGSEqD0Z0FgIF1m+fDnTp08vd3lycjLe3t4YjUYAvvvuO+Lj4wHIyMigd+/edOrUiY0bNwIwY8YMFi1axPr167n55psBKCgoYOjQoWRmZvLLL7+g1+sdr/vKK6/wxBNPOL2mqqooikJSUhI9evQgODiYAwcOOMrwzTffcM899/Df//6XW2+9tY6PiBB1T2pqQrjYvHnz6NixY6n5BoPB8bxXr16OQAMIDg5m0qRJfPjhh2RmZmI0Gvnuu+/o0aOHI9AAfHx8eOihh3juuec4fPgwffv2Zd26dRiNRh5++OFSr6koitP02LFjHYEGMHDgQAASExNr+naFcCkJNSFcrHfv3pU2FGnXrl25886dO4fRaOT8+fNl3tPrWmCeO3eOvn37cvbsWdq3b+8UmuUpeb+vawGXmZlZ6bZCeAJpKCKEcNBqtWXOV1W5SiEaBgk1ITzQ6dOny50XHR0NQFRUFAkJCaXWO3nypNN6bdq04dSpU5jN5voqrhAeQ0JNCA908OBB9u7d65jOyMjgiy++oH///o5TgqNGjeLw4cP89NNPjvVMJhNLly4lIiKCnj17AvbrZJmZmbz//vulXkdqYKKxkWtqQrjYDz/8wJkzZ0rN79OnD+3btwcgLi6OyZMnM3XqVEeT/tzcXGbOnOlY/8knn2T16tVMnjzZqUn/8ePH+fDDD9Hp7H/ed911FytXrmTmzJkcPHiQgQMHYjKZ2LlzJxMmTOCuu+5yzRsXwgUk1IRwsblz55Y5/5///Kcj1Pr378/NN9/M3LlzSUxMpH379ixfvpxBgwY51g8LC2Pjxo3MmjWLJUuWUFBQQOfOnfnkk0+cGpBotVo+//xz3n77bVatWsWGDRsICgqib9++jtqcEI2F9FMTwsMYjUamTJkiI+YLUQNyTU0IIUSjIaEmhBCi0ZBQE0II0WhIQxEhPIyM3iFEzUlNTQghRKMhoSaEEKLRkFATQgjRaEioCSGEaDQk1IQQQjQaEmpCCCEajf8fjeD12qaPy/8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\"\"\"可视化下看看训练情况\"\"\"\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('Model loss')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Test'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "united-estonia",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T14:38:07.688257Z",
     "start_time": "2021-05-07T14:38:07.536616Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20/20 [==============================] - 0s 19ms/sample - loss: 0.8741 - binary_crossentropy: 0.8741 - auc: 0.5267\n",
      "val AUC: 0.874087\n"
     ]
    }
   ],
   "source": [
    "\"\"\"模型评估\"\"\"\n",
    "print('val AUC: %f' % model.evaluate(valx_model_input, valy, batch_size=batch_size)[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "automatic-example",
   "metadata": {},
   "source": [
    "## 模型架构"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "automatic-grass",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T14:38:24.184295Z",
     "start_time": "2021-05-07T14:38:22.095829Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed to import pydot. You must install pydot and graphviz for `pydotprint` to work.\n"
     ]
    }
   ],
   "source": [
    "from tensorflow import keras\n",
    "keras.utils.plot_model(model, to_file='./AutoInt_arc.png', show_shapes=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
